# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from pyquery import PyQuery as pq
import re
from scrapy.http import Request as sreq
from scrapy.http import FormRequest as fr
from girl.items.dmm import *
from scrapy.shell import inspect_response
import requests as req
import time
import datetime
class AvgirlSpider(CrawlSpider):
    name = "dmmgirl"
    # allowed_domains = ["dmm.co.jp"]

    start_urls =[
        "http://www.dmm.co.jp/digital/videoa/-/actress/=/keyword=a/",
        "http://www.dmm.co.jp/digital/nikkatsu/-/actress/=/keyword=ka/"
        ]
    rules = (
        # Rule(LinkExtractor(allow=('.*keyword.*'),restrict_xpaths="//ul[@class='d-item']//a"),follow=True,callback="girl"),
        Rule(LinkExtractor(allow=('.*keyword.*'),restrict_xpaths="//ul[@class='d-modtab']//a"),follow=True,callback="girl2"),
        Rule(LinkExtractor(allow=('.*keyword.*'),restrict_xpaths="//ul[@class='d-modsort-la']//a"),follow=True,callback="girl2"),
        # Rule(LinkExtractor(allow=('.*page.*'),restrict_xpaths="//div[@class='list-boxcaptside list-boxpagenation group']//a"),callback="girl"),
        # Rule(LinkExtractor(allow=('.*page.*'),restrict_xpaths="//div[@class='list-boxcaptside']//a",follow=False),
    )
    custom_settings = {
        'RETRY_ENABLED' : False,
        "CONCURRENT_REQUESTS":20,
        # "HTTPCACHE_IGNORE_MISSING":True
    }
    def girl(self,response):
        item = avgirlItem()
        x = response.xpath
        tmp = x("//ul[@class='d-item act-box-100 group']/li")
        for y in tmp:
            item["thumb"] = y.xpath("a/img/@src").extract_first()
            item["url"] = y.xpath("a/@href").extract_first()
            item["letter"] = y.xpath("a/span/text()").extract_first()
            item["rkey"] = item["letter"][0]
            item["type"] = "av"
            item["name"] = y.xpath("a/text()").extract_first()
            item["key"] = re.search("keyword=(\w+)",response.url).groups(0)[0]
            yield item
    def girl2(self,response):
        item = avgirlItem()
        x = response.xpath
        tmp = x("//ul[@class='act-box-65 group mg-b20']/li")
        for y in tmp:
            item["thumb"] = y.xpath("a/img/@src").extract_first()
            item["url"] = y.xpath("a/@href").extract_first()
            item["letter"] = ""
            item["rkey"] = ""
            item["type"] = "film"
            item["name"] = y.xpath("a/text()").extract_first()
            item["key"] = re.search("keyword=(\w+)",response.url).groups(0)[0]
            yield item
